﻿#region ... Copyright Notice ...
/*
   Copyright 2008 Tyler Jensen

	Author: Tyler Jensen    http://www.tsjensen.com/blog

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 */
#endregion

using System;
using System.Data;
using System.Collections.Generic;
using System.Configuration;
using System.Linq;
using System.Web;
using System.Xml.Linq;
using Atrax.Contracts;
using Atrax.Library;
using Atrax.Consumers;
using Atrax.Security;
using Atrax.Logging;
using Atrax.Utils;

namespace Atrax.Spider
{
	internal static class SpiderController
	{
		static object locker = new object();
		
		/// <summary>
		/// key: spiderQueryId
		/// </summary>
		static Dictionary<string, RequestHandler> handlersList = new Dictionary<string, RequestHandler>();

		/// <summary>
		/// keys: pacingHost
		/// </summary>
		static Dictionary<string, int> inProcessCount = new Dictionary<string, int>();

		/// <summary>
		/// keys: pacingHost
		/// </summary>
		static Dictionary<string, double> minPacingHostSeconds = new Dictionary<string, double>();

		/// <summary>
		/// keys: pacingHost
		/// </summary>
		static Dictionary<string, List<IpAddressRecord>> ipAddressRecord = new Dictionary<string, List<IpAddressRecord>>();

		/// <summary>
		/// keys: pacingHost
		/// </summary>
		static Dictionary<string, Queue<string>> checkoutRequests = new Dictionary<string, Queue<string>>();

		/// <summary>
		/// keys: pacingHost
		/// </summary>
		static Dictionary<string, int> lastCheckoutIndex = new Dictionary<string, int>();

		static string[] ipAddressPool = null;

		internal static SpiderResponse EnqueRequest(SpiderRequest request)
		{
			lock (locker)
			{
				SpiderResponse response = null;
				try
				{
					Uri requestUri = new Uri(request.Request.Url);
					double requiredSecondsBetweenRequests = request.Request.PacingSecondsPerIpAddress;
					bool pacingRequired = requiredSecondsBetweenRequests == 0.0 ? false : true;
					string pacingHost = pacingRequired ? requestUri.Host.ToLower() : "none";

					//attend to host pacing management
					if (!minPacingHostSeconds.ContainsKey(pacingHost))
					{
						minPacingHostSeconds.Add(pacingHost, requiredSecondsBetweenRequests);
					}
					else
					{
						//update value if this value is greater
						//later when complete, check to see if any of these pacingHost requests are queue and take to 0.0
						//to assure that the value remains as small as possible
						if (minPacingHostSeconds[pacingHost] < requiredSecondsBetweenRequests) minPacingHostSeconds[pacingHost] = requiredSecondsBetweenRequests;
					}

					//add default of 2 seconds for GET'ing each request
					double estimatedDeliverSeconds = GetRequestsInQueueSamePacingHost(pacingHost) * (requiredSecondsBetweenRequests + 2); 

					response = new SpiderResponse()
					{
						SpiderQueryId = Guid.NewGuid().ToString(),
						Request = request,
						TimeRequestMade = DateTime.UtcNow,
						EstimatedDeliverySeconds = estimatedDeliverSeconds,
						StatusCode = "200",
						StatusDescription = "OK"
					};

					RequestHandler requestHandler = new RequestHandler(request, response.SpiderQueryId, pacingHost, requiredSecondsBetweenRequests, pacingRequired);
					handlersList.Add(requestHandler.SpiderQueryId, requestHandler); //keeps reference to query until done and removed by call from query

					//enque query (will kick off worker on worker thread)
					ManagedThreadPool.QueueUserWorkItem(requestHandler.ThreadPoolCallback, requestHandler);

					//update inProcessCount
					if (!inProcessCount.ContainsKey(pacingHost))
						inProcessCount.Add(pacingHost, 1);
					else
						inProcessCount[pacingHost]++;
				}
				catch (Exception e)
				{
					response = new SpiderResponse()
					{
						Request = request,
						TimeRequestMade = DateTime.UtcNow,
						EstimatedDeliverySeconds = 0.0,
						StatusCode = "501",
						StatusDescription = e.Message
					};
				}
				return response;
			}
		}

		/// <summary>
		/// Only used on a cold restart by the master.
		/// </summary>
		internal static void KillAllJobs()
		{
			lock (locker)
			{
				handlersList.Clear();
				ManagedThreadPool.Reset();
			}
		}

		/// <summary>
		/// Used to remove query from querys list for counts. Called by the query when it's done.
		/// </summary>
		/// <param name="m_spiderQueryId"></param>
		internal static void CompleteJob(string spiderQueryId, string pacingHost, string ipAddress)
		{
			lock (locker)
			{
				Log.Debug(string.Format("+++ SpiderController.CompleteJob called for spider query id {0}", spiderQueryId));
				if (ipAddressRecord.ContainsKey(pacingHost))
				{
					foreach (IpAddressRecord rec in ipAddressRecord[pacingHost])
					{
						if (rec.IpAddress == ipAddress)
						{
							rec.IsCheckedOut = false;
							rec.LastUsed = DateTime.Now;
						}
					}
				}
				handlersList.Remove(spiderQueryId); //now we can GC the handler

				//now check to see if minPacingHostSeconds can be set to 0.0
				inProcessCount[pacingHost]--;
				if (inProcessCount[pacingHost] < 1) minPacingHostSeconds[pacingHost] = 0.0; //reset to minimum
			}
		}

		/// <summary>
		/// Used by the query to get a checkout token ("take a number") for use in the CheckoutIpAddress
		/// </summary>
		/// <param name="pacingHost"></param>
		/// <returns></returns>
		internal static CheckoutToken GetIpAddressCheckoutToken(string pacingHost)
		{
			lock (locker)
			{
				if (ipAddressPool == null) ipAddressPool = ConfigurationManager.AppSettings["ipAddressPool"].Split(',');
				if (!ipAddressRecord.ContainsKey(pacingHost))  //initialize if not there
				{
					ipAddressRecord.Add(pacingHost, new List<IpAddressRecord>()); //initialize list
					lastCheckoutIndex.Add(pacingHost, 0); //initialize last checked out index - always start with 0
					foreach (string ip in ipAddressPool)
					{
						ipAddressRecord[pacingHost].Add(new IpAddressRecord(ip, DateTime.MinValue));
					}
				}
				CheckoutToken token = new CheckoutToken() { Token = Guid.NewGuid().ToString() };
				if (!checkoutRequests.ContainsKey(pacingHost)) checkoutRequests.Add(pacingHost, new Queue<string>());
				checkoutRequests[pacingHost].Enqueue(token.Token);
				return token;
			}
		}

		/// <summary>
		/// Will return an ip address if one is available otherwise returns a null.
		/// </summary>
		/// <param name="pacingHost"></param>
		/// <param name="token"></param>
		/// <param name="pacingSeconds"></param>
		/// <returns></returns>
		internal static string CheckoutIpAddress(string pacingHost, CheckoutToken token)
		{
			lock (locker)
			{
				double pacingSeconds = minPacingHostSeconds.ContainsKey(pacingHost) ? minPacingHostSeconds[pacingHost] : 0.0;
				string ipAddress = null;
				if (checkoutRequests.ContainsKey(pacingHost) && checkoutRequests[pacingHost].Count > 0 && checkoutRequests[pacingHost].Peek() == token.Token) //is it my turn
				{
					int nextCheckoutIndex = lastCheckoutIndex[pacingHost] + 1;
					if (nextCheckoutIndex > ipAddressRecord[pacingHost].Count - 1) nextCheckoutIndex = 0;

					if (nextCheckoutIndex > 0)
					{
						//start with nextCheckoutIndex then go from 0 to < nextCheckoutIndex 
						for (int i = nextCheckoutIndex; i < ipAddressRecord[pacingHost].Count; i++)
						{
							IpAddressRecord rec = ipAddressRecord[pacingHost][i];
							ipAddress = GetIpAddressIfAvailable(rec, pacingHost, pacingSeconds);
							if (ipAddress != null)
							{
								lastCheckoutIndex[pacingHost] = i; //note for next time
								break;
							}
						}
						if (ipAddress == null) //only continue in loop if not found
						{
							for (int i = 0; i < nextCheckoutIndex; i++)
							{
								IpAddressRecord rec = ipAddressRecord[pacingHost][i];
								ipAddress = GetIpAddressIfAvailable(rec, pacingHost, pacingSeconds);
								if (ipAddress != null)
								{
									lastCheckoutIndex[pacingHost] = i; //note for next time
									break;
								}
							}
						}
					}
					else
					{
						//loop through 0 - end
						for (int i = 0; i < ipAddressRecord[pacingHost].Count; i++)
						{
							IpAddressRecord rec = ipAddressRecord[pacingHost][i];
							ipAddress = GetIpAddressIfAvailable(rec, pacingHost, pacingSeconds);
							if (ipAddress != null)
							{
								lastCheckoutIndex[pacingHost] = i; //note for next time
								break;
							}
						}
					}
				}
				return ipAddress;
			}
		}

		static string GetIpAddressIfAvailable(IpAddressRecord rec, string pacingHost, double pacingSeconds)
		{
			string ipAddress = null;
			//auto checkin any address in use over 3 minutes
			if (rec.IsCheckedOut)
			{
				TimeSpan ts = DateTime.Now - rec.LastUsed;
				if (ts.TotalMinutes >= 3.0) rec.IsCheckedOut = false; //automatically checked back in
			}
			if (!rec.IsCheckedOut)  //if not checked out and pacing is good, then check out the address
			{
				TimeSpan ts = DateTime.Now - rec.LastUsed;
				if (ts.TotalSeconds >= pacingSeconds)
				{
					rec.IsCheckedOut = true;
					rec.LastUsed = DateTime.Now;
					ipAddress = rec.IpAddress;
					checkoutRequests[pacingHost].Dequeue(); //pull off queue if address found and checked out
					Log.Debug(string.Format("*** SpiderController.CheckoutIpAddress called returned {0}", ipAddress));
				}
			}
			return ipAddress;
		}

		static int GetRequestsInQueueSamePacingHost(string pacingHost)
		{
			int total = 0;
			foreach (KeyValuePair<string, RequestHandler> item in handlersList)
			{
				if (item.Value.PacingHost == pacingHost) total++;;
			}
			return total;
		}
	}

	internal class CheckoutToken
	{
		public string Token { get; set; }
	}
}
